BLM data analysis

Installing required packages

library(rtweet)

package 㤼㸱rtweet㤼㸲 was built under R version 4.0.4

library(dplyr)
library(tidyr)
library(tidytext)

package 㤼㸱tidytext㤼㸲 was built under R version 4.0.4

library(ggplot.multistats)

package 㤼㸱ggplot.multistats㤼㸲 was built under R version 4.0.4

library(ggplot2)

package 㤼㸱ggplot2㤼㸲 was built under R version 4.0.4

library(forestmangr)

package 㤼㸱forestmangr㤼㸲 was built under R version 4.0.4
Attaching package: 㤼㸱forestmangr㤼㸲

The following object is masked from 㤼㸱package:data.table㤼㸲:

    :=

library(syuzhet)

package 㤼㸱syuzhet㤼㸲 was built under R version 4.0.4
Attaching package: 㤼㸱syuzhet㤼㸲

The following object is masked from 㤼㸱package:rtweet㤼㸲:

    get_tokens

library(tidyverse)

package 㤼㸱tidyverse㤼㸲 was built under R version 4.0.4Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
-- Attaching packages ------------------------------------------------------------------------------------------------------------ tidyverse 1.3.0 --
v tibble  3.0.4     v stringr 1.4.0
v readr   1.4.0     v forcats 0.5.0
v purrr   0.3.4     
package 㤼㸱stringr㤼㸲 was built under R version 4.0.4-- Conflicts --------------------------------------------------------------------------------------------------------------- tidyverse_conflicts() --
x dplyr::between()   masks data.table::between()
x dplyr::filter()    masks stats::filter()
x dplyr::first()     masks data.table::first()
x purrr::flatten()   masks rtweet::flatten()
x dplyr::lag()       masks stats::lag()
x dplyr::last()      masks data.table::last()
x purrr::transpose() masks data.table::transpose()

library(readxl) # read excel

package 㤼㸱readxl㤼㸲 was built under R version 4.0.4

library(tibble) # tobble dataframe
library(dplyr) # piping
library(stringr) # character manipulation
library(tidytext)
library(tokenizers)

package 㤼㸱tokenizers㤼㸲 was built under R version 4.0.4

library(stopwords)

package 㤼㸱stopwords㤼㸲 was built under R version 4.0.4

library(tidyverse)
library(lobstr)

package 㤼㸱lobstr㤼㸲 was built under R version 4.0.4

library(glue)

Attaching package: 㤼㸱glue㤼㸲

The following object is masked from 㤼㸱package:dplyr㤼㸲:

    collapse

library(stringr)

Loading the data

blm2020<-read.csv('../../gen/data-preparation/temp/BLM2020_dataset.csv', sep = ' ', na.strings=c("", "NA"))

blm2021<-read.csv('../../gen/data-preparation/temp/BLM2021_dataset.csv', sep = ' ', na.strings=c("", "NA"))

EOF within quoted stringError in read.table(file = file, header = header, sep = sep, quote = quote,  : 
  duplicate 'row.names' are not allowed

Show the ratio of replies, retweets and likes

#Retweets
blm2020_retweets<-blm2020[blm2020$retweet > 1, ]
blm2020_retweets

blm2021_retweets<-blm2021[blm2021$retweet > 1, ]
blm2021_retweets

NA

#Likes
blm2020_likes<-blm2020[blm2020$like >1, ]
blm2020_likes

blm2021_likes<-blm2021[blm2021$like >1, ]
blm2021_likes

NA

#Replies
blm2020_replies<-blm2020[blm2020$reply >1, ]
blm2020_replies

blm2021_replies<-blm2021[blm2021$reply >1, ]
blm2021_replies

NA

Creating a data frame

data2020<-data.frame(category=c("Retweets", "Likes", "Replies"), count=c(2, 5, 3))
data2020

data2021<-data.frame(category=c("Retweets", "Likes", "Replies"), count=c(707, 1738, 574))
data2021

Visualizing the data 2020

#Adding columns
data2020$fraction = data2020$count / sum(data2020$count)
data2020$percentage = data2020$count / sum(data2020$count) * 100
data2020$ymax = cumsum(data2020$fraction)
data2020$ymin = c(0, head(data2020$ymax, n=-1))

#Rounding up

data2020 <- round_df(data2020, 2)

# Specify what the legend should say
Type_of_Tweet <- paste(data2020$category, data2020$percentage, "%")

ggplot(data2020, aes(ymax=ymax, ymin=ymin, xmax=4, xmin=3, fill=Type_of_Tweet)) +
  geom_rect() +
  coord_polar(theta="y") + 
  xlim(c(2, 4)) +
  theme_void() +
  theme(legend.position = "right")

Visualizing the data 2021

#Adding columns
data2021$fraction = data2021$count / sum(data2021$count)
data2021$percentage = data2021$count / sum(data2021$count) * 100
data2021$ymax = cumsum(data2021$fraction)
data2021$ymin = c(0, head(data2021$ymax, n=-1))

#Rounding up

data2021 <- round_df(data2021, 2)

# Specify what the legend should say
Type_of_Tweet <- paste(data2020$category, data2021$percentage, "%")

ggplot(data2021, aes(ymax=ymax, ymin=ymin, xmax=4, xmin=3, fill=Type_of_Tweet)) +
  geom_rect() +
  coord_polar(theta="y") + 
  xlim(c(2, 4)) +
  theme_void() +
  theme(legend.position = "right")

Most frequent words in the 2020 tweets

Lets see which words are more frequently used in the posts of 2020

blm2020$text <- gsub("@\\S*", "", blm2020$text)
blm2020$text <- gsub("amp", "", blm2020$text) 
blm2020$text <- gsub("[\r\n]", "", blm2020$text)
blm2020$text <- gsub("[[:punct:]]", "", blm2020$text)

#removing stop words from the text - so the most frequent words are not or at and

tweets <- blm2020 %>%
  select(text) %>%
  unnest_tokens(word, text)
tweets <- tweets %>%
  anti_join(get_stopwords(language = "nl", source = "snowball"))

Joining, by = "word"

#making a histogram
tweets %>% # gives you a bar chart of the most frequent words found in the tweets
  count(word, sort = TRUE) %>%
  top_n(15) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
  labs(y = "Count",
       x = "Unique words",
       title = "Most frequent words found in the tweets of #BLM",
       subtitle = "Stop words removed from the list")

Selecting by n

Most frequent words in the 2021 tweets Lets now see which words are more frequently used in the posts of 2021

blm2021$text <- gsub("@\\S*", "", blm2021$text)
blm2021$text <- gsub("amp", "", blm2021$text) 
blm2021$text <- gsub("[\r\n]", "", blm2021$text)
blm2021$text <- gsub("[[:punct:]]", "", blm2021$text)

#removing stop words from the text - so the most frequent words are not or at and

tweets1 <- blm2021 %>%
  select(text) %>%
  unnest_tokens(word, text)
tweets1 <- tweets1 %>%
  anti_join(get_stopwords(language = "nl", source = "snowball"))

Joining, by = "word"

#making a histogram
tweets1 %>% # gives you a bar chart of the most frequent words found in the tweets
  count(word, sort = TRUE) %>%
  top_n(15) %>%
  mutate(word = reorder(word, n)) %>%
  ggplot(aes(x = word, y = n)) +
  geom_col() +
  xlab(NULL) +
  coord_flip() +
  labs(y = "Count",
       x = "Unique words",
       title = "Most frequent words found in the tweets of #BLM 2021",
       subtitle = "Stop words removed from the list")

Selecting by n

Sentiment analysis for positive or negative tweets 2020

# Converting tweets to ASCII to trackle strange characters
tweets_sentiment <- iconv(tweets, from="UTF-8", to="ASCII", sub="")

# removing retweets, in case needed 
tweets_sentiment <-gsub("(RT|via)((?:\\b\\w*@\\w+)+)","",tweets_sentiment)

# removing mentions, in case needed
tweets_sentiment <-gsub("@\\w+","",tweets_sentiment)
ew_sentiment<-get_nrc_sentiment((tweets_sentiment))
sentimentscores<-data.frame(colSums(ew_sentiment[,]))
names(sentimentscores) <- "Score"
sentimentscores <- cbind("sentiment"=rownames(sentimentscores),sentimentscores)
rownames(sentimentscores) <- NULL
ggplot(data=sentimentscores,aes(x=sentiment,y=Score))+
  geom_bar(aes(fill=sentiment),stat = "identity")+
  theme(legend.position="none")+
  xlab("Sentiments")+ylab("Scores")+
  ggtitle("Total sentiment based on scores 2020")+
  theme_minimal()

Sentiment analysis for positive or negative tweets 2021

# Converting tweets to ASCII to trackle strange characters
tweets_sentiment_1 <- iconv(tweets, from="UTF-8", to="ASCII", sub="")

# removing retweets, in case needed 
tweets_sentiment_1 <-gsub("(RT|via)((?:\\b\\w*@\\w+)+)","",tweets_sentiment_1)

# removing mentions, in case needed
tweets_sentiment_1 <-gsub("@\\w+","",tweets_sentiment_1)
ew_sentiment<-get_nrc_sentiment((tweets_sentiment))
sentimentscores<-data.frame(colSums(ew_sentiment[,]))
names(sentimentscores) <- "Score"
sentimentscores <- cbind("sentiment"=rownames(sentimentscores),sentimentscores)
rownames(sentimentscores) <- NULL
ggplot(data=sentimentscores,aes(x=sentiment,y=Score))+
  geom_bar(aes(fill=sentiment),stat = "identity")+
  theme(legend.position="none")+
  xlab("Sentiments")+ylab("Scores")+
  ggtitle("Total sentiment based on scores 2021")+
  theme_minimal()

Visualizing the variables in the dataset 2020

#Making a ggplot
#ploting retweets by location
ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = retweet, y = location))

#ploting retweets by datetime
#need to separate date from time
ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = retweet, y = datetime), color = "blue")

Graphing users amount of followers and user amount friends

ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = user_amount_followers, y = user_amount_friends), color = "blue")

Graphing users amount status by location To check how much users Tweet by Location

ggplot(data = blm2020) + 
  geom_point(mapping = aes(x = user_amount_status, y = location), color = "blue")

#Filter if parties are mentioned in 2020

state.name[grep ("VVD", "PvdA", "PVV", "CDA", "SP", "CU", "PvdD", state.name)]

Visualizing the variables in the dataset 2021 ####needs amendments!!!!

#Making a ggplot
#ploting retweets by location
ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = retweet, y = location))

#ploting retweets by datetime
#need to separate date from time
ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = retweet, y = datetime), color = "blue")

Graphing users amount of followers and user amount friends

ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = user_amount_followers, y = user_amount_friends), color = "blue")

Graphing users amount status by location To check how much users Tweet by Location

ggplot(data = blm2021) + 
  geom_point(mapping = aes(x = user_amount_status, y = location), color = "blue")

#Filter if parties are mentioned in 2021

state.name[grep ("VVD", "PvdA", "PVV", "CDA", "SP", "CU", "PvdD", state.name)]

#remive dollar signs

blm2020<-gsub("\\$", "", blm2020)
blm2021<-gsub("\\$","", blm2021)

Sentiment analysis

# create a list of tokens
#https://www.kaggle.com/rtatman/tokenization-tutorial

tokens_2020<-data.frame(text= blm2020) %>% unnest_tokens(word, text)
tokens_2020

tokens_2021<-data.frame(text=blm2021) %>% unnest_tokens(word, text)
tokens_2021

Now that we have a list of tokens, we need to compare them against a list of words with either positive or negative sentiment.

# get the sentiment from the first text: 
tokens_2020 %>%
  inner_join(get_sentiments("bing")) %>% # pull out only sentiment words
  count(sentiment) %>% # count the # of positive & negative words
  spread(sentiment, n, fill = 0) %>% # made data wide rather than narrow
  mutate(sentiment = positive - negative) # # of positive words - # of negative words

Joining, by = "word"

#do the same for 2021

tokens_2021 %>%
  inner_join(get_sentiments("bing")) %>% # pull out only sentiment words
  count(sentiment) %>% # count the # of positive & negative words
  spread(sentiment, n, fill = 0) %>% # made data wide rather than narrow
  mutate(sentiment = positive - negative) # # of positive words - # of negative words

Joining, by = "word"

blm2020$sentiment<-get_sentiment(blm2020$text)

Error in blm2020$text : $ operator is invalid for atomic vectors

Now we will perform an ANOVA with DV the sentiment of the text and control variablw the source of the tweet

https://www.scribbr.com/statistics/anova-in-r/

#2020
one.way_2020 <- aov(sentiment ~ source_tweet , data = blm2020)

Error in model.frame.default(formula = sentiment ~ source_tweet, data = blm2020,  : 
  'data' must be a data.frame, environment, or list

#2021

one.way_2021 <- aov(sentiment ~ source_tweet, data = blm2021)

Error in model.frame.default(formula = sentiment ~ source_tweet, data = blm2021,  : 
  'data' must be a data.frame, environment, or list

#How can we measure how many tweets are made by location?

references: 1. https://towardsdatascience.com/a-guide-to-mining-and-analysing-tweets-with-r-2f56818fdd16 2. https://ourcodingclub.github.io/tutorials/time/ 3. https://cran.r-project.org/web/packages/stopwords/stopwords.pdf 4. https://www.dummies.com/programming/r/how-to-search-for-individual-words-in-r/ 5. https://r4ds.had.co.nz/data-visualisation.html 6. Anova and sentiment analysis: https://www.kaggle.com/rtatman/tutorial-sentiment-analysis-in-r

sum(""(BLM2020))
sum(is.na(BLM2020$Datetime))
sum(is.na(BLM2020$Text))
sum(is.na(BLM2020$Tweet.Id))
sum(is.na(BLM2020$Username))
sum(is.na(BLM2020$Location))
sum(is.na(BLM2020$Location))
summary(BLM2020)
summary(BLM2020$Location)

regression retweets

retweets_glm <-glm(Retweet ~ ., data=BLM2020)

Error in is.data.frame(data) : object 'BLM2020' not found

One way ANOVA

#removing html tags from the source_tweet variable

library(textclean)
html_tags <- c(
    "<bold>Random</bold> text with symbols: &nbsp; &lt; &gt; &amp; &quot; &apos;",
    "<p>More text</p> &cent; &pound; &yen; &euro; &copy; &reg;"
)

blm2020$source_tweet<- replace_html(blm2020$source_tweet)

Error in blm2020$source_tweet : $ operator is invalid for atomic vectors

running the test

#creating a 

one.way <- aov(source_tweet ~ ., data = blm2020)

Error in terms.formula(formula, "Error", data = data) : 
  '.' in formula and no 'data' argument

#Graphs for every variable #Graph on user ammount of followers #Check if a user has made multiple tweets - how can we do that? #Filter if parties are mentioned